{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b9583f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/xiongjiedai\n",
      "/Users/xiongjiedai/llama.cpp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xiongjiedai/Library/Python/3.9/lib/python/site-packages/IPython/core/magics/osm.py:393: UserWarning: using bookmarks requires you to install the `pickleshare` library.\n",
      "  bkms = self.shell.db.get('bookmarks', {})\n"
     ]
    }
   ],
   "source": [
    "%cd ~\n",
    "%cd llama.cpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4f651103-a6c8-4b7d-9f0f-be0553b22acf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m13B\u001b[m\u001b[m                              ggml-vocab-gpt-neox.gguf\n",
      "\u001b[34m13B-v2\u001b[m\u001b[m                           ggml-vocab-llama.gguf\n",
      "\u001b[34m30B\u001b[m\u001b[m                              ggml-vocab-mpt.gguf\n",
      "\u001b[34m7B\u001b[m\u001b[m                               ggml-vocab-refact.gguf\n",
      "\u001b[34m7B-v2\u001b[m\u001b[m                            ggml-vocab-stablelm-3b-4e1t.gguf\n",
      "ggml-vocab-aquila.gguf           ggml-vocab-starcoder.gguf\n",
      "ggml-vocab-baichuan.gguf         \u001b[31mtokenizer.model\u001b[m\u001b[m\n",
      "ggml-vocab-falcon.gguf\n"
     ]
    }
   ],
   "source": [
    "# obtain the original LLaMA model weights and place them in ./models\n",
    "!ls ./models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b991d454",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # build the GGUF read/write example\n",
    "# !make gguf\n",
    "\n",
    "# # write a dummy GGUF model to test.gguf\n",
    "# !./gguf test.gguf w\n",
    "\n",
    "# # read the dummy GGUF model\n",
    "# !./gguf test.gguf r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a57dd4e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # install Python dependencies\n",
    "# !python3 -m pip install -r requirements.txt\n",
    "\n",
    "# # llama 2\n",
    "# # convert the model to ggml FP16 format (edit your params.json file if the \"vocab_size\" mismatch)\n",
    "# !python3 convert.py models/7B-v2/\n",
    "# !python3 convert.py models/13B-v2/\n",
    "\n",
    "# # quantize the model to 4-bits (using q4_0 method)\n",
    "# !./quantize ./models/7B-v2/ggml-model-f16.gguf ./models/7B-v2/ggml-model-q4_0.gguf q4_0\n",
    "# !./quantize ./models/13B-v2/ggml-model-f16.gguf ./models/13B-v2/ggml-model-q4_0.gguf q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8184a10f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I llama.cpp build info: \n",
      "I UNAME_S:   Darwin\n",
      "I UNAME_P:   arm\n",
      "I UNAME_M:   arm64\n",
      "I CFLAGS:    -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread \n",
      "I CXXFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi \n",
      "I NVCCFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread    -Wno-pedantic -Xcompiler \"-Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi \"\n",
      "I LDFLAGS:   -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "I CC:        Apple clang version 15.0.0 (clang-1500.0.40.1)\n",
      "I CXX:       Apple clang version 15.0.0 (clang-1500.0.40.1)\n",
      "\n",
      "rm -vrf *.o tests/*.o *.so *.dll benchmark-matmult common/build-info.cpp *.dot *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead tests/test-c.o metal tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope tests/test-backend-ops\n",
      "build-info.o\n",
      "common.o\n",
      "console.o\n",
      "ggml-alloc.o\n",
      "ggml-backend.o\n",
      "ggml-metal.o\n",
      "ggml-quants.o\n",
      "ggml.o\n",
      "grammar-parser.o\n",
      "llama.o\n",
      "sampling.o\n",
      "train.o\n",
      "tests/test-c.o\n",
      "benchmark-matmult\n",
      "common/build-info.cpp\n",
      "main\n",
      "quantize\n",
      "quantize-stats\n",
      "perplexity\n",
      "embedding\n",
      "vdot\n",
      "q8dot\n",
      "train-text-from-scratch\n",
      "convert-llama2c-to-ggml\n",
      "simple\n",
      "batched\n",
      "batched-bench\n",
      "save-load-state\n",
      "server\n",
      "gguf\n",
      "llama-bench\n",
      "libllava.a\n",
      "llava-cli\n",
      "baby-llama\n",
      "beam-search\n",
      "speculative\n",
      "infill\n",
      "tokenize\n",
      "parallel\n",
      "finetune\n",
      "export-lora\n",
      "lookahead\n",
      "metal\n",
      "I llama.cpp build info: \n",
      "I UNAME_S:   Darwin\n",
      "I UNAME_P:   arm\n",
      "I UNAME_M:   arm64\n",
      "I CFLAGS:    -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread \n",
      "I CXXFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi \n",
      "I NVCCFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread    -Wno-pedantic -Xcompiler \"-Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi \"\n",
      "I LDFLAGS:   -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "I CC:        Apple clang version 15.0.0 (clang-1500.0.40.1)\n",
      "I CXX:       Apple clang version 15.0.0 (clang-1500.0.40.1)\n",
      "\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread    -c ggml.c -o ggml.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c llama.cpp -o llama.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/common.cpp -o common.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/sampling.cpp -o sampling.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/grammar-parser.cpp -o grammar-parser.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/console.cpp -o console.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread  -c ggml-metal.m -o ggml-metal.o\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread    -c ggml-alloc.c -o ggml-alloc.o\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread    -c ggml-backend.c -o ggml-backend.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread     -c ggml-quants.c -o ggml-quants.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/train.cpp -o train.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -Wunreachable-code-break -Wunreachable-code-return -Wdouble-promotion -pthread  -c tests/test-c.c -o tests/test-c.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -c common/build-info.cpp -o build-info.o\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  pocs/vdot/vdot.cpp ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o vdot -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  pocs/vdot/q8dot.cpp ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o q8dot -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/benchmark/benchmark-matmult.cpp build-info.o ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o benchmark-matmult -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/export-lora/export-lora.cpp ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o export-lora -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/metal/metal.cpp ggml.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o metal -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/main/main.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o console.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o main -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/quantize/quantize.cpp build-info.o ggml.o llama.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o quantize -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/quantize-stats/quantize-stats.cpp build-info.o ggml.o llama.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o quantize-stats -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/perplexity/perplexity.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o perplexity -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/embedding/embedding.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o embedding -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o train-text-from-scratch -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o convert-llama2c-to-ggml -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/simple/simple.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o simple -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/batched/batched.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o batched -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/batched-bench/batched-bench.cpp build-info.o ggml.o llama.o common.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o batched-bench -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/save-load-state/save-load-state.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o save-load-state -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -Iexamples/server examples/server/server.cpp examples/llava/clip.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o server -framework Accelerate -framework Foundation -framework Metal -framework MetalKit   -Wno-cast-qual\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/gguf/gguf.cpp ggml.o llama.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o gguf -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/llama-bench/llama-bench.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o llama-bench -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  -static -fPIC -c examples/llava/llava.cpp -o libllava.a -Wno-cast-qual\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/llava/llava-cli.cpp examples/llava/clip.cpp examples/llava/llava.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o llava-cli -framework Accelerate -framework Foundation -framework Metal -framework MetalKit  -Wno-cast-qual\n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/baby-llama/baby-llama.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o baby-llama -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/beam-search/beam-search.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o beam-search -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/speculative/speculative.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o speculative -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/infill/infill.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o console.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o infill -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/tokenize/tokenize.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o tokenize -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/parallel/parallel.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o parallel -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/finetune/finetune.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o finetune -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "c++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_DARWIN_C_SOURCE -DNDEBUG -DHAVE_BUGGY_APPLE_LINKER -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64 -DGGML_USE_METAL  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi  examples/lookahead/lookahead.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-metal.o ggml-alloc.o ggml-backend.o ggml-quants.o -o lookahead -framework Accelerate -framework Foundation -framework Metal -framework MetalKit \n",
      "\n",
      "====  Run ./main -h for help.  ====\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# metal build\n",
    "!make clean && LLAMA_METAL=1 make -j"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17c21608",
   "metadata": {},
   "source": [
    "# inference"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b6aee37",
   "metadata": {},
   "source": [
    "### 7B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ae626ed6-adc9-429b-9f04-c1fb2cfd8b13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702016403\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 3647.97 MiB\n",
      "llm_load_tensors: mem required  = 3647.97 MiB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "llama_new_context_with_model: max tensor size =   102.54 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  3647.98 MiB, ( 3653.69 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   256.03 MiB, ( 3909.72 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    70.52 MiB, ( 3980.23 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under this head, I shall give two examples out of many which might be cited of the easy flow of her luxurious current through the heart of any city you please to mention, from Athens and Nineveh only excepted of ancient cities. \n",
      " The first was a small one: in some parts of our island it would pass without remark; in others it was gigantic – I mean, if the fact of its being so small went for anything in such places as Brummagem or Bednal Green, where nothing but gigantical is capable of going unregarded. It was simply this: That the single little street of Old N—-, that one dear little old-fashioned line of houses with the most delightful conservatory and front gardens, should be called Hatchard Street, after Richardson's shop there! \n",
      " The second example was a greater one. I give it for what it has to do with my story – very little as yet: but still something. I will relate it in the words of that noble critic, Dr. Burney, himself an eye-witness of all three of the scenes of which he treats; and will not make him any more unreasonable than his own facts are liable to be called. \n",
      " He was going with some friends to take a view of Paris from one of its lofty steeples: and it was a very hot day in July or August, I forget which. But the fact is certain that at noon they got into a coach at a spot which has ever since been called 'The Place Louis Quinze', on the opposite side to where it stands now; for the whole site of Paris is continually changing, like that other great city we know better, whose sites are in a great measure the same as of old. \n",
      " But they had not gone far – very little indeed, if I remember right, before Dr. Burney, from a sudden impulse (I suppose he must have been looking out at some distant object of interest), cried 'Stop!' and immediately alighted out of the coach again into the dusty street, where no carriage was to be seen; and not so much as a hackney-coach, though there were two or three such in sight. \n",
      " Then turning about (said he) as if I were going back for my portmanteau (as I had never been at Paris before, nor the least idea what was proper to be done on such an occasion), I went into one of the houses near the coach-stand; and after inquiring the way to M. de Tilly's house, in a few minutes got there in safety.\n",
      "Thus he turned aside from his great object, that of taking a view of Paris from its steeple, and alighted instead into some obscure inn (I forget its name), where he and his friends remained for two or three hours; but the point which struck him most at this house was, the innumerable multitude of flies. I do not think Dr. Burney ever after spoke a word to me on the subject of that fly-stand of the Maison Tilly, but it is very possible that he did; for there are many things in his talk which remain unrecalled upon my memory, while others which seem more striking at first, have become quite lost and forgotten in time.\n",
      "Dr. Burney, in short, was an observer of minute details, and never so much as suspected himself of any tendency to omit any little particular circumstance that happened under his notice; nor, on the other hand, did he ever speak of anything which had not been well considered by him beforehand (and often afterward too).\n",
      "In my own opinion, however, Dr. Burney never saw things as they were in reality at all: I have no doubt that every thing was true which he told me; but then his perceptions and impressions were so false that it made no sort of difference to the reality at bottom what they might be: for instance, if a man were to see a whole city burning before him from the summit of some hill, where there is only a little blaze in any other part; such a man would never know how the city was really set on fire. It would be no use for him to look at the blazing houses on this side and on that side of the flames, with their shining roofs; he would always see only one house burning as the most striking object before his eyes: just so did Dr. Burney always see a great city before him, though all its parts were not really set on fire at once.\n",
      "These were my thoughts about it when I was first a little acquainted with Dr. Burney; but afterwards, I found that this habitual vision of one object only was the very character and peculiarity of his perceptions. For\n",
      "llama_print_timings:        load time =    1409.92 ms\n",
      "llama_print_timings:      sample time =      67.28 ms /  1024 runs   (    0.07 ms per token, 15220.88 tokens per second)\n",
      "llama_print_timings: prompt eval time =    2676.20 ms /   494 tokens (    5.42 ms per token,   184.59 tokens per second)\n",
      "llama_print_timings:        eval time =   49029.79 ms /  1023 runs   (   47.93 ms per token,    20.86 tokens per second)\n",
      "llama_print_timings:       total time =   51891.80 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ef52530a-c041-4855-8d88-a20e63dff79b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702016456\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 3647.97 MiB\n",
      "llm_load_tensors: mem required  = 3647.97 MiB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "llama_new_context_with_model: max tensor size =   102.54 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  3647.98 MiB, ( 3653.69 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   256.03 MiB, ( 3909.72 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    70.52 MiB, ( 3980.23 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her sprightly ministry of resources, she had put her financial system on such a sound basis, that for some time past it has kept every body else at work, except herself: and now this beautiful system has been recommended by a member of our all-powerful British parliament as \"a model to the world.\" It is, indeed, perhaps, a model to the world, and has nothing to do with the shape of Plutus' horse. \n",
      " England is not in debt, except to herself; but is able and ready to lend or borrow, as occasion may require: turning her supplies into taxes, or her demands into notes: working upon those who work for her, or upon other people's stock of labour; or producing that effect by working on herself. It has been an axiom with the statesman, for many years back, that interest is a principle that governs all minds, and it would be well if our legislature could get a little credit in that way. But this interests no body. It is considered as not altogether unworthy to notice here, that some people object to taxes as too much like work. \n",
      " The most successful rival of the present age (successful by reason of being really and truly so) has been a system of society, founded on these ideas—\"that all riches are derived from industry; that the greatest good luck is merely a great capacity for taking advantage of it, when it comes in our way.\" \n",
      " CHAPTER XI.\n",
      "IN WHICH THE READER IS SHOWN, THAT IT MAY BE UNDERSTOOD IN THE CORRECT LIGHT TO WHICH IT OWES ITS EFFECTS OF GREATNESS—WHEN HE HAS DONE SO, AN ATTENTION TO A FEW OF THE PRINCIPLES THROUGHOUT CONTAINED IN IT WILL MAKE THOSE EFFECTS VERY CLEAR; AND A BRIEF STATEMENT OF OTHER PRINCIPLES CONTINUALLY AT WORK, WHICH PREPARED THE MIND FOR SUCH GREATNESS.\n",
      "THESE are the principles upon which a nation may be founded, and in which it can maintain its ground. It is not necessary to the greatness of a country that she should have been built up by the labour of her own people; for this would only show, that they had no other means of subsistence; whereas, if that people had such natural talents as enabled them to work out and establish any kind of manufacture which could be made profitable on their own territory, it is equally necessary, in order to their national greatness, that she should not have been built up by her own people.\n",
      "But when it comes into a nation's head that there are greater means of subsistence than those which are naturally hers; that there are means of subsisting herself on better terms with other nations; that the country from whence she draws these means will allow them to be had on the most profitable and advantageous terms; that this can be done, not by having recourse to her natural talents and industry for raising the money wherewith she is to pay for those means; but merely by having recourse to her natural talents and industry in raising a sum of money sufficient for building up, maintaining, and defending herself against her neighbours; that such sum can be had without labour on their own part; but merely by applying themselves with great assiduity to their business in the usual manner; that this is done by making use of these means with no other view than that which is intended for the advancement of trade; that this is also done by a further means, consisting in the payment being made out of the very produce or products from whence the money has been obtained:—this idea, I say, coming into the head of any country, that her subsistence shall be raised with greater ease and advantage, is equally necessary to make her great., whether this be done by means of money only; or if it consists in applying these other means (as consisting in making use of a currency and bullion which have been drawn from abroad); or even if they consist in raising up the value of a manufacture already existing among themselves.\n",
      "But when, instead of having recourse to any of those means which may be hereafter pointed out for establishing trade with other nations; these other means being used by herself as a part of her own natural industry and talents in subsisting and supporting herself on better terms with the rest of the world; that is, merely upon such a footing as this: That if any nation can raise up such a manufacture among themselves, which may be made profitable upon their own soil; and also make use of all those\n",
      "llama_print_timings:        load time =     803.19 ms\n",
      "llama_print_timings:      sample time =      70.67 ms /  1024 runs   (    0.07 ms per token, 14490.70 tokens per second)\n",
      "llama_print_timings: prompt eval time =    2677.62 ms /   494 tokens (    5.42 ms per token,   184.49 tokens per second)\n",
      "llama_print_timings:        eval time =   49329.55 ms /  1023 runs   (   48.22 ms per token,    20.74 tokens per second)\n",
      "llama_print_timings:       total time =   52199.53 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4de933c0-53d0-4fd4-9b98-724d2ebbd2d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702016509\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 3647.97 MiB\n",
      "llm_load_tensors: mem required  = 3647.97 MiB\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "llama_new_context_with_model: max tensor size =   102.54 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  3647.98 MiB, ( 3653.69 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   256.03 MiB, ( 3909.72 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    70.52 MiB, ( 3980.23 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m \n",
      " Under the pressure of the spirit of conquest in that Age of Light, a thing like a large wheel borne with incredible laboriousness over divers difficult hills and dales by a band of individuals who were chiefly distinguished for their universal desire to avoid all trouble of any kind, especially mental exertion, was set on fire, the flame thereof being reflected from the burning paper roll up in the air, to light many a cottage in the humble valley where Wesley and Whitfield walked. Then slowly, majestically moving from the burning paper, the travelers of the Wheel began to sing! It was their own national air.\n",
      "For many years that air had been familiar in the great cities of the Old World; for it was the air oft-times sung by those who contemplated and desired reformation or revolution in the State of Society. The time came at last, when all men turned to look for the spiritual bright shiners which should be given then out of the treasures of earth that they might set fire to the darkness around; and so it was, as if by enchantment, that from out of the air there spread a lovely rainbow! But this lovely arc was no rainbow of hope: it was a rainbow which had never yet been seen.\n",
      "In this loveliest of all mornings, this softest and sweetest of all dawns, arose an army with banners; but their banners were not like unto those which men have made for themselves in times of warring. These banners bore a device—a device which had never yet been seen by mortal eyes. The first man who saw it was one whose life had been given wholly to study of the past and present states of the world, and to thought concerning its future destiny: and his name was Condorcet. He saw this device—this lovely rainbow—as he passed along the Rue St Honore, in a carriage in which were many ladies; for it was not permitted that any one should walk upon these streets on that day, except where such permission had been granted by authority, and then only within certain limits. But this man saw further than any of those who walked beside him or passed along the road. For he beheld in the loveliest of all mornings the new era which was to dawn, even though his vision might be clouded for a space!\n",
      "Then rose another army; but it was no armed host. It bore banners as before, each of them bearing some device, or device and word, or words and devices: but these were such things as men have made by themselves in days gone by, and never since then. And the first man who saw this new era was one whose life had been given wholly to study of past and present states of society, and to thought concerning its future destiny—and his name was Saint Simon; for it is not permitted that any one should walk upon these streets on such days except those who have been specially allowed. But even he saw not the fuller vision. For he only beheld in the loveliest of all mornings what man might see, but the truth and certainty of which could never be proved to him by human evidence: the fact that in the new era there was one who would lead forth men from their present ignorance and blindness into light and knowledge; that he would bring them out of error into Truth!\n",
      "Then rose a third army, which bore with it a strange banner. For the device upon it was no visible symbol or object which man could make: but an emanation of the heart within the mind, or the soul within the body—of some truth which may be felt and known only in the inner part of our being. And the first man who saw this new era was one whose life had been given wholly to study of past and present states of society, and to thought concerning its future destiny: his name is Hegel; for it is not permitted that any one should walk upon these streets on such days except those who have been specially allowed. But even he saw not the fuller vision; for though in the new era there would be a man of Truth and Knowledge, this was the man himself.\n",
      "But how can all this be?\n",
      "When men had seen God in human shape they began to think that some other than the human shape which is known to them might possibly contain Him: such as the Shape without form; the Infinite; or even—though it were a mad and blasphemous thought—a shape like unto the Divine! Then they came to see how unknowable the divine essence is in itself, because of its absolute perfection. And thus they began to think that all else might be unknowable, except through the light which He imparts from within their\n",
      "llama_print_timings:        load time =     796.58 ms\n",
      "llama_print_timings:      sample time =      70.29 ms /  1024 runs   (    0.07 ms per token, 14569.05 tokens per second)\n",
      "llama_print_timings: prompt eval time =    2678.28 ms /   494 tokens (    5.42 ms per token,   184.45 tokens per second)\n",
      "llama_print_timings:        eval time =   49257.67 ms /  1023 runs   (   48.15 ms per token,    20.77 tokens per second)\n",
      "llama_print_timings:       total time =   52128.09 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f07d6cd",
   "metadata": {},
   "source": [
    "### 7B F16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "540fbaab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702016563\n",
      "llama_model_loader: loaded meta data with 15 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.12 MiB\n",
      "llm_load_tensors: mem required  = 12853.12 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "llama_new_context_with_model: max tensor size =   250.00 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  8192.00 MiB, offs =            0\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  4911.14 MiB, offs =   8327774208, (13108.84 / 10922.67)ggml_metal_add_buffer: warning: current allocated size is greater than the recommended max working set size\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   256.03 MiB, (13364.88 / 10922.67)ggml_metal_add_buffer: warning: current allocated size is greater than the recommended max working set size\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    70.52 MiB, (13435.39 / 10922.67)ggml_metal_add_buffer: warning: current allocated size is greater than the recommended max working set size\n",
      "ggml_metal_graph_compute: command buffer 2 failed with status 5\n",
      "GGML_ASSERT: ggml-metal.m:1872: false\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7eaf89e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702016584\n",
      "llama_model_loader: loaded meta data with 15 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.12 MiB\n",
      "llm_load_tensors: mem required  = 12853.12 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors she entertained herself, while Mark Antony read her Sonnets over the dead Caesar; she practised ostentation both in eating and in drinking; and she revived and brought back into dangerous activity the ancient factions with their deadly energies, the exact tendency and nature of which remain at this day to be seen. The venal spirit of the times was exploited by foolish ideologues of the right and left who set out in search of imaginary vast conspiracies against liberty, and ended in being themselves slaves in a despotism more galling than Egypt ever knew, or Rome in her palmiest days. \n",
      " It was under these same influences that my adventures first began. Amid the confused noise of human passions and pursuits, I had withdrawn myself from society to seek rest in the silence of the cloister: but this retreat was soon disturbed. An eccentric, half-distracted clergyman had recently obtained possession of a house in a valley near Combe-Ash; where he established himself as a hermit, living solitary and abstinent among the mountains which hemmed him on every side. He had been accustomed to pass much time upon the high peak called Hangingstone; and it was noticed that for some few months before his final retreat to his house of solitude, he often indulged himself in a promenade down this wild mountain track, by means of a horse which was found at times tied up behind a cave close under the summit. There was something very unsocial in all these habits; and I believe there were many who thought him little better than mad—if indeed he could be called so; for certainly his pursuits were of an order the most rational and prudent. He had been accustomed to seek rest at the periodical meetings of certain ancient religious fraternities, which had at times met in England and other countries on this side of the world. I was well acquainted with these institutions: my father had been a member of them during his earlier years; and when he returned to England from some foreign land, there were often a few of our family who kept up their old habits of fraternisation. I believe it would not be easy to name any body of men so widely diffused as the brotherhood of this order of the ancient Catholic Church: they have subsisted in various parts of Europe for many centuries; and I am convinced there are at present a very large number who are united together by certain habits of mutual friendship, or fraternity. They were chiefly persons whose profession was to devote themselves to study, and the acquirement of that knowledge which should enable them to do their duty in this world as they ought; but I believe there were many who had not so much time at command for the pursuit of such studies: these were more generally termed “obscure” persons by the more learned members. There was one ancient book, in which the tenets of the order and the various customs of its members were preserved; but it was believed that a knowledge even of this was not necessary to be considered as belonging to the society: indeed, I believe that a great part of those who now keep up these associations do so upon no other ground than their belief in certain doctrines which have been long taught by them. The brotherhood had become, however, so much disorganised and so mixed up with the superstitions of the middle ages, that they were regarded at this period chiefly as a relic of barbarous and unreasonable belief; and certainly there were not wanting persons who thought it an improvement to mankind that these old associations should be broken down. In other instances, however, such was the love of learning for its own sake which had been so long fostered by this society, that even when its members separated from each other in search after knowledge, their pursuits were no longer disjoined and independent—they took the same route and were animated by the same feelings; and thus a very powerful association continued to be formed amongst them. It is remarkable, as showing how different opinions are connected with different persons in the same state of society, that there was scarcely an instance recorded in which any brotherhood, however venerable its traditions or high its rank, was preserved from being overwhelmed and lost by those associations. But whilst this fact shows what is sometimes called a “_moral influence_,” it has also another very powerful tendency; for when we consider the great number of men who have devoted themselves to learning, in any age or country whatever, we find that even such persons as were not connected together by any bond of common interests must yet be linked to each other by some such feelings as those which arise from a similar taste. I do not mean to say that all men who\n",
      "llama_print_timings:        load time =   14419.03 ms\n",
      "llama_print_timings:      sample time =     227.75 ms /  1024 runs   (    0.22 ms per token,  4496.12 tokens per second)\n",
      "llama_print_timings: prompt eval time =   15687.77 ms /   494 tokens (   31.76 ms per token,    31.49 tokens per second)\n",
      "llama_print_timings:        eval time =  253992.80 ms /  1023 runs   (  248.28 ms per token,     4.03 tokens per second)\n",
      "llama_print_timings:       total time =  270573.42 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# try cpu\n",
    "!./main --color --no-mmap -ngl 0 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "47a3b477",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702016869\n",
      "llama_model_loader: loaded meta data with 15 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.12 MiB\n",
      "llm_load_tensors: mem required  = 12853.12 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she entertained herself, besides, with such humane achievements as sentencing a youth to have his hands cut off, his tongue torn out by the roots, and his body burned alive: all which he had contrived to perform upon himself, while suspended in chains, prior to being convicted and carried into execution. It was under this Christian reign, that the Duke of Orleans, second son to Louis XVI., having contracted a clandestine marriage with one Mademoiselle de la Motte, was declared an outlaw by his father's decree; and forthwith, having found means to land himself in England, requesting asylum for him and his consort at Windsor. Here the couple were honourably received: the king receiving him as a subject without ceremonies, on his own door-step: and Lady Suffolk as her own child; treating them with every mark of favour till it should please Heaven to restore her sister's family to their rightful rank. \n",
      "It will have been seen that I have now arrived at the period when our first friend began to attract public observation, under circumstances which rendered him an early favourite with his own class and with many who had never before taken any notice of a poetical character. He was as well received in private life as he was distinguished in society. His manners were easy and pleasant; and though the great number of guests whom he entertained might, by some persons, be considered as approaching too much to affectation or extravagance, they rather served to show the universal esteem in which his character and conduct were held, than as a proof that it was either affected or extravagant. At any rate, these parties afforded him no other reproach but that of being rather too hospitable; for which offence I never knew any one to charge him with having given them occasion. But to return from this digression to the course of my story: \n",
      "CHAPTER IV\n",
      "THE poet's first publications were two satires, entitled, _The Dying Christian_ and _The Complaint_. The former was occasioned by a popular cant, in which many who died without confessing themselves, were supposed notwithstanding to have been Christians. In the other poem he has taken up a question that had at this time been long agitated among the dissenters: whether those who died as unbaptized infants might be saved by any of their parents having been members in the true church; or must die without hope, however they might live piously and devoutly to the last. The poet's sentiments on this point were, that these persons _might_ be saved; but, considering it as a matter of uncertainty, he recommended them not to rely upon any such uncertain hope, but rather endeavour so to conduct themselves, as that their dying in a state of nature might prove no barrier to the reception they should meet with from Jesus Christ.\n",
      "These publications were received very favourably; and, at one time or other, all the booksellers, except one only, gave him to understand that he was engaged with them for an edition of three thousand copies each of both poems; which, if it had taken place, would have been a fine fortune.\n",
      "His next publication was _The Spleen_ , which was considered as one of the best compositions he had produced, and the booksellers all gave him to understand that it was sure of success. It contained about fourteen hundred lines; and in one of the prefaces his editor acknowledged with truth, 'that some of the expressions used were not within their usual style.'\n",
      "He now resolved upon publishing an original poem on a large scale. The design had been in his head ever since he first began to write; but at that time it was not within the power of human industry and ingenuity so far to extend the limits of poetry as they could be afterwards, by means of the steam-engine and other discoveries that were then on the point of being made.\n",
      "He resolved that it should consist entirely of new materials; neither shall I here mention the name or subject of his intended poem, because in a future chapter you will learn how the original was destroyed, and a substitute given instead.\n",
      "The design being now formed, he next took up an old copy of Hudibras, which he had several years before purchased, with a view to some of these very purposes that have since been carried into execution; and he began a systematic selection from this poem for his new composition; and the booksellers were all unanimously ready to give him such encouragement as their best abilities could afford.\n",
      "His scheme was laid so completely open to them, that it might well be thought, by some readers of this memoir, that they might have been\n",
      "llama_print_timings:        load time =   14339.41 ms\n",
      "llama_print_timings:      sample time =      71.87 ms /  1024 runs   (    0.07 ms per token, 14247.75 tokens per second)\n",
      "llama_print_timings: prompt eval time =   15624.73 ms /   494 tokens (   31.63 ms per token,    31.62 tokens per second)\n",
      "llama_print_timings:        eval time =  250783.49 ms /  1023 runs   (  245.15 ms per token,     4.08 tokens per second)\n",
      "llama_print_timings:       total time =  266657.75 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# try cpu\n",
    "!./main --color --no-mmap -ngl 0 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702017150\n",
      "llama_model_loader: loaded meta data with 15 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight f16      [  4096, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight f16      [  4096,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight f16      [ 11008,  4096,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight f16      [  4096, 11008,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  4096,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = mostly F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 12853.12 MiB\n",
      "llm_load_tensors: mem required  = 12853.12 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.57 MiB\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors she entertained herself, besides, with such humane achievements as sentencing a youth to have his eyes burned out with hot plates (for stealing a handkerchief) or hanging a man for trying to make himself useful by forcing the gate through which his prison escaped him. The question was so delicate that it reached the throne itself for decision, and received it: very wisely too, considering the trembling situation of things in France; for nothing tends more to give a royal power permanency than a universal belief in its infallibility. \n",
      " Yet she too had an Empress who stretched her hand, if only metaphorically, over many kingdoms. For, as I have observed already, it is not only from Asia, but from Africa we derive the ivory tusks of the hippopotamus. They were, in fact, one of the first luxuries imported by Bonaparte on his first entry into Marseilles; and there are stones set in the handsome bridges over the Seine at Paris which came out of the same quarter. It was no vain boast of yours truly to declare himself an African, with an ideal right of dominating both Asia and America. Both continents were mine by the title of Lord Curzon, K. C. B.; but there is something in the idea of being an African even better than the mere truth: for the idea includes a certain largeness of mind. \n",
      " I had always felt an interest in the life of the Emperor Napoleon III. and was rather sorry at the way he disappeared from public notice on his death, as if we were not entitled to know much about him; so that I set myself to look into it a little. It seemed to me that, while every other European personage, male or female, had been recorded in a biography, our own eminent person had gone out of print and could only be traced by the specialist: like some old book which has long ceased to interest any one but the student of bibliography. I suppose we must feel this way about him because he was English; for it would have been different if he had not been an Englishman, or a Frenchman, but a German. His being so must be set down in large measure to the fact that he came of royal blood. He was, indeed, by the title of his wife, a son-in-law of Queen Victoria; but for this relationship I can well believe that neither France nor any other country would ever have looked on him as a hero. And it is only the English who ever think in terms of family: not even of nation or race. The French and Germans might never have heard of him except as a great man, who happened to be a son-in-law; whereas, with us, he has become one of the greatest names in English history. This was not because we liked him better than other men, but because, through his wife, we are related to him. I cannot believe that it is the same feeling which exists about any great German who might have married into an English family: as, for instance, when George III. became King of Hanover. It would never have occurred to us then to think of his son-in-law in terms of history: because we had not thought much of our own king's relation; and probably we should never have heard about it at all except that he died under circumstances which made the news more interesting than usual, so that it reached the public. \n",
      " This was something which I tried to get a view on when I began to think of writing this book: whether we were bound in conscience to write it about him, or whether, because he was related to us in that way and died under strange conditions, the English should take him upon their shoulders as a great man. I asked several people whom I respected; some said they could not understand why, on such an issue, the English should bother themselves at all. But that was exactly what we were doing: making him a hero. The Germans would never have done it for a man who married into their family and died under strange conditions; because, in spite of his relation with them by marriage, they did not think of him as one of themselves. On the other hand, they might take any of us English who married into German families and died under equally strange circumstances; so that, whatever happened to him, he would be a hero there: although we should never have heard of it. \n",
      "And it is curious how such things work out in practice. As soon as the Germans invaded Belgium they found they had no need to worry about our reaction at all: because they did not care whether we were shocked or not: because we could not do anything to them. We had lost our power to prevent invasion: and that was how they looked on it; so that, while there\n",
      "llama_print_timings:        load time =    9520.59 ms\n",
      "llama_print_timings:      sample time =      71.46 ms /  1024 runs   (    0.07 ms per token, 14328.89 tokens per second)\n",
      "llama_print_timings: prompt eval time =   39490.38 ms /   494 tokens (   79.94 ms per token,    12.51 tokens per second)\n",
      "llama_print_timings:        eval time =  374754.41 ms /  1023 runs   (  366.33 ms per token,     2.73 tokens per second)\n",
      "llama_print_timings:       total time =  414528.88 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "# try cpu\n",
    "!./main --color --no-mmap -ngl 0 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74a84502",
   "metadata": {},
   "source": [
    "### 13B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "43038979-9395-4119-94c8-5a8b90198064",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702017575\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  291:             blk.32.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  292:             blk.32.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  293:             blk.32.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  294:        blk.32.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  295:           blk.32.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  296:           blk.32.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  297:             blk.32.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  298:          blk.32.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  299:           blk.32.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  300:             blk.33.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  301:             blk.33.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  302:             blk.33.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  303:        blk.33.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  304:           blk.33.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  305:           blk.33.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  306:             blk.33.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  307:          blk.33.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  308:           blk.33.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  309:             blk.34.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  310:             blk.34.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  311:             blk.34.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  312:        blk.34.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  313:           blk.34.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  314:           blk.34.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  315:             blk.34.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  316:          blk.34.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  317:           blk.34.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  318:             blk.35.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  319:             blk.35.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  320:             blk.35.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  321:        blk.35.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  322:           blk.35.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  323:           blk.35.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  324:             blk.35.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  325:          blk.35.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  326:           blk.35.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  327:             blk.36.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  328:             blk.36.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  329:             blk.36.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  330:        blk.36.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  331:           blk.36.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  332:           blk.36.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  333:             blk.36.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  334:          blk.36.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  335:           blk.36.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  336:             blk.37.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  337:             blk.37.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  338:             blk.37.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  339:        blk.37.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  340:           blk.37.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  341:           blk.37.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  342:             blk.37.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  343:          blk.37.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  344:           blk.37.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  345:             blk.38.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  346:             blk.38.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  347:             blk.38.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  348:        blk.38.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  349:           blk.38.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  350:           blk.38.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  351:             blk.38.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  352:          blk.38.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  353:           blk.38.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  354:             blk.39.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  355:             blk.39.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  356:             blk.39.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  357:        blk.39.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  358:           blk.39.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  359:           blk.39.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  360:             blk.39.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  361:          blk.39.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  362:           blk.39.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 7024.03 MiB\n",
      "llm_load_tensors: mem required  = 7024.03 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 78.07 MiB\n",
      "llama_new_context_with_model: max tensor size =   128.17 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  7024.03 MiB, ( 7029.73 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   400.03 MiB, ( 7429.77 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    75.02 MiB, ( 7504.78 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastor, she christened her leading men of state \"the thirty.\" It was a fine story in those days, when, by some mysterious fatality not to be deplored, some ten or twelve natives of the republic had fallen into the sea from the deck of a ship. There was much weeping for them at their homes in Paris; and their friends lamented far more over their sudden disappearance than the people of England lament for those who are annually swallowed up by old Father Thames, alive or dead, and whose names are speedily forgotten. \n",
      " But what shall be said to this? That, if the liberation of one people was the benefit which the other derived from the slavery of another, surely there must have been more good done by the slave states than ever existed in the whole existence of the slave system. That, supposing for the sake of a abstract argument that things in no wise beneficial shall be held to be truly so, the most humble and oppressed negro has exerted a greater influence in favor of human progress than can be attributed to any individual of his white oppressor or enslaver. That, if Christianity is a fact, we must look for it among such men as angels could watch over; not among the half-apish, half-human figures who in our day have been called slaves. And finally, that from the slave states—not New England—the Puritan and Puritans sprang. \n",
      " CHAPTER XXVIII.  \n",
      "COTTON\n",
      "WITHIN a fortnight after the events related in the last chapter, the old man was ready for his voyage to the South. It is unnecessary to detail all the minutiae of preparation; it will suffice to say that on the 6th of July Mr. George Wilmot came with all speed from Boston, and took up his quarters at the Mansion-House; and that before he had been a week in residence, the old man was safely lodged under the shadow of St. John's steeple. He looked but poorly about him. The sight of this new home, after his years of wanderings, could not fail to be impressive upon an untutored mind; and he sat for hours in one place, staring with a kind of stupid wonder at the walls of his narrow chamber, at the black oak bedsteads, which were so much more commodious than those of old times, when one's head was left to hang out over the end. He could not sleep there; but slept in the great garret-loft, upon a mattress on the floor—there being no bed—and all the night through he kept his face turned towards the light that shone through the high gable window: \"the stars looked so much more glorious!\" He made inquiry for one Mr. Shelton, in the neighboring township; but he was not to be found.\n",
      "On the following day—which was Sunday—the old man was at early morn carried over from his home, and set up, with all reverence, on the green before St. John's. It must have been a novel spectacle, this white-haired man sitting there upon his rude chair; and he looked more like an ancient statue than one of living flesh. But when he began to speak—a very strange old voice it was that fell from those thin lips—the people stood amazed; for he seemed to be no longer in the body at all, but wholly in the spirit; a voice without its substance! The preacher had come over from Strawberry Hill with his congregation on this holy day. His sermon was of the old time; not about the love of Christ which passeth knowledge, nor even of the cross of Christ, but about \"the wrath to come.\" And as he spoke, the old man rose and began to move among the people who sat in their pews before him; speaking at times, sometimes only with his eyes: so that it was as if one should hear two sermons. Then there came a silence, while the preacher made a prayer for this day of grace and rest: whereupon arose the old man, and he was wondrously beautiful in all the radiance of his gray hair; and the people stood awestruck, for they saw that he had not heard any part of what had been said.\n",
      "The old man came to the pulpit; and lifting up his hands above him—\"My brethren,\" he cried aloud, \"whatsoever things are true!\" He lifted his hand above himself again: \"Ye know in your own hearts that it is so!\"\n",
      "He did not sit down from that pulpit; but stood there like some figure\n",
      "llama_print_timings:        load time =    2836.41 ms\n",
      "llama_print_timings:      sample time =      70.54 ms /  1024 runs   (    0.07 ms per token, 14516.17 tokens per second)\n",
      "llama_print_timings: prompt eval time =    5088.78 ms /   494 tokens (   10.30 ms per token,    97.08 tokens per second)\n",
      "llama_print_timings:        eval time =   90892.87 ms /  1023 runs   (   88.85 ms per token,    11.26 tokens per second)\n",
      "llama_print_timings:       total time =   96177.02 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5de46ea1-0750-477e-9183-d1e6e0cc9e2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702017674\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  291:             blk.32.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  292:             blk.32.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  293:             blk.32.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  294:        blk.32.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  295:           blk.32.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  296:           blk.32.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  297:             blk.32.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  298:          blk.32.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  299:           blk.32.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  300:             blk.33.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  301:             blk.33.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  302:             blk.33.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  303:        blk.33.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  304:           blk.33.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  305:           blk.33.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  306:             blk.33.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  307:          blk.33.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  308:           blk.33.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  309:             blk.34.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  310:             blk.34.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  311:             blk.34.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  312:        blk.34.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  313:           blk.34.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  314:           blk.34.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  315:             blk.34.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  316:          blk.34.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  317:           blk.34.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  318:             blk.35.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  319:             blk.35.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  320:             blk.35.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  321:        blk.35.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  322:           blk.35.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  323:           blk.35.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  324:             blk.35.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  325:          blk.35.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  326:           blk.35.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  327:             blk.36.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  328:             blk.36.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  329:             blk.36.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  330:        blk.36.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  331:           blk.36.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  332:           blk.36.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  333:             blk.36.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  334:          blk.36.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  335:           blk.36.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  336:             blk.37.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  337:             blk.37.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  338:             blk.37.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  339:        blk.37.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  340:           blk.37.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  341:           blk.37.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  342:             blk.37.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  343:          blk.37.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  344:           blk.37.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  345:             blk.38.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  346:             blk.38.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  347:             blk.38.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  348:        blk.38.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  349:           blk.38.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  350:           blk.38.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  351:             blk.38.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  352:          blk.38.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  353:           blk.38.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  354:             blk.39.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  355:             blk.39.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  356:             blk.39.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  357:        blk.39.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  358:           blk.39.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  359:           blk.39.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  360:             blk.39.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  361:          blk.39.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  362:           blk.39.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 7024.03 MiB\n",
      "llm_load_tensors: mem required  = 7024.03 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 78.07 MiB\n",
      "llama_new_context_with_model: max tensor size =   128.17 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  7024.03 MiB, ( 7029.73 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   400.03 MiB, ( 7429.77 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    75.02 MiB, ( 7504.78 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of its noble author, the British nation did the same; and its Courts of Justice took off the hat to Mr. Madison, and canvassed the merits of Cobbenton as a field for growing hemp. \"United we stand, divided we fall,\" is a true word among many other political truisms equally true; and if the British Empire and the American Republic should both go down in actual hatlessness to French-rooted hell or to German-planted heaven, this memorable pair of platitudes will be remembered and quoted by garlic-eating Huns, and shallot-munching Gauls alike.\n",
      "Having a weakness for Latin quotations, Mr. Thomas Carlyle in the course of his history relates that on one occasion when some matter was under discussion, and the French ambassador had been referred to as a \"fellow with a great head,\" Lord Palmerston observed that \"the Frenchman has a very large skull, but not a large head.\" Upon which Mr. Carlyle adds, that therein lies the whole secret of modern France. But this is carrying our history rather too far afield into biology, and so beyond the scope of our present theme; and in any case we need hardly apprehend from our own personal experience that any human skull can be considered great at all, unless it be first well filled.\n",
      "Now I wish to call your attention for a moment to these two facts: that a man's skull has as its limit his brains; and that the size of the one is limited by the capacity of the other. What do you think the head of a man should contain, if it be great? Why not the whole world, I suppose. Or even the whole universe. And who knows but some day in this modern world all that will not be found possible to do,—or even probable.\n",
      "What an achievement for us in these latter days; and what a triumph! But before that can happen let us pause a moment longer at this point, and ponder on the meaning of all this. We have been speaking of the great head. We know how very great it can be in actuality. Now I am going to tell you of what I conceive is its purpose in existence.\n",
      "Now then, if the universe itself shall not prove a sufficient world for the brain of man—I will say that when the time shall come to him to make another, he may have to begin by pulling down all the walls of heaven; and then it would be just as well that he should also have with him the very chisel-edge tools which he shall use in constructing anew these same walls. But whether we call it the world or a universe, it is surely enough for us even now to make one if we will but set our minds to the doing of it; and the thing will grow of its own accord until at last it shall fill the whole space that is available for it in the universe itself.\n",
      "This will be possible to do when man has once made a beginning at constructing his world, just as surely as the sun shines now, or the stars burn steadily away; but just because the same man is the one who builds it, he must first of all build his own brain. This then becomes his first universe to deal with. But by-and-by, when this too has become overcrowded—for even so can happen;—then, and not until then, let him make another world for himself; which, again, will grow until it fills the whole space that is available for it in the second universe. This done, if he be but patient enough to wait the proper time for it, the third universe will come of itself as a matter of course. And thus on, ever on—or at least so long as man may live and build!\n",
      "The world too, you see, must grow. But because this growth is in him, man grows with it; whereas, if the thing be done by some other power outside of himself, he does not grow with the same result, but rather to a more or less undesirable diminution of himself as an entity apart from the world, and in which he becomes wholly lost. For, even as he cannot know too much, so neither can it be any good for him that his universe should grow too fast; otherwise all his labor is in vain. But this may also happen:—The world will not stay still! It too must go on growing, or at least so long as man may live and build!\n",
      "Then, I say, let the thing begin by constructing a brain for himself first of all. And when that has become overcrowded, then he should make another one: which again will fill its own space in no time—and so on to the very last of it; or at least so long as\n",
      "llama_print_timings:        load time =    2261.12 ms\n",
      "llama_print_timings:      sample time =     166.71 ms /  1024 runs   (    0.16 ms per token,  6142.26 tokens per second)\n",
      "llama_print_timings: prompt eval time =    5094.60 ms /   494 tokens (   10.31 ms per token,    96.97 tokens per second)\n",
      "llama_print_timings:        eval time =   93335.84 ms /  1023 runs   (   91.24 ms per token,    10.96 tokens per second)\n",
      "llama_print_timings:       total time =   98798.12 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "763256c8-af9e-493e-b52d-4c161814bd72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1620 (fe680e3)\n",
      "main: built with Apple clang version 15.0.0 (clang-1500.0.40.1) for arm64-apple-darwin23.1.0\n",
      "main: seed  = 1702017776\n",
      "llama_model_loader: loaded meta data with 16 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: - tensor    0:                token_embd.weight q4_0     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    1:               output_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor    2:                    output.weight q6_K     [  5120, 32000,     1,     1 ]\n",
      "llama_model_loader: - tensor    3:              blk.0.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    4:              blk.0.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    5:              blk.0.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    6:         blk.0.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    7:            blk.0.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor    8:            blk.0.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor    9:              blk.0.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   10:           blk.0.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   11:            blk.0.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   12:              blk.1.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   13:              blk.1.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   14:              blk.1.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   15:         blk.1.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   16:            blk.1.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   17:            blk.1.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   18:              blk.1.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   19:           blk.1.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   20:            blk.1.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   21:              blk.2.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   22:              blk.2.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   23:              blk.2.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   24:         blk.2.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   25:            blk.2.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   26:            blk.2.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   27:              blk.2.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   28:           blk.2.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   29:            blk.2.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   30:              blk.3.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   31:              blk.3.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   32:              blk.3.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   33:         blk.3.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   34:            blk.3.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   35:            blk.3.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   36:              blk.3.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   37:           blk.3.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   38:            blk.3.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   39:              blk.4.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   40:              blk.4.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   41:              blk.4.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   42:         blk.4.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   43:            blk.4.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   44:            blk.4.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   45:              blk.4.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   46:           blk.4.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   47:            blk.4.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   48:              blk.5.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   49:              blk.5.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   50:              blk.5.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   51:         blk.5.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   52:            blk.5.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   53:            blk.5.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   54:              blk.5.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   55:           blk.5.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   56:            blk.5.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   57:              blk.6.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   58:              blk.6.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   59:              blk.6.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   60:         blk.6.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   61:            blk.6.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   62:            blk.6.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   63:              blk.6.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   64:           blk.6.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   65:            blk.6.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   66:              blk.7.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   67:              blk.7.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   68:              blk.7.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   69:         blk.7.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   70:            blk.7.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   71:            blk.7.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   72:              blk.7.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   73:           blk.7.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   74:            blk.7.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   75:              blk.8.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   76:              blk.8.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   77:              blk.8.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   78:         blk.8.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   79:            blk.8.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   80:            blk.8.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   81:              blk.8.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   82:           blk.8.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   83:            blk.8.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   84:              blk.9.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   85:              blk.9.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   86:              blk.9.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   87:         blk.9.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   88:            blk.9.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   89:            blk.9.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   90:              blk.9.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   91:           blk.9.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   92:            blk.9.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor   93:             blk.10.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   94:             blk.10.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   95:             blk.10.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   96:        blk.10.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   97:           blk.10.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor   98:           blk.10.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor   99:             blk.10.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  100:          blk.10.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  101:           blk.10.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  102:             blk.11.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  103:             blk.11.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  104:             blk.11.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  105:        blk.11.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  106:           blk.11.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  107:           blk.11.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  108:             blk.11.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  109:          blk.11.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  110:           blk.11.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  111:             blk.12.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  112:             blk.12.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  113:             blk.12.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  114:        blk.12.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  115:           blk.12.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  116:           blk.12.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  117:             blk.12.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  118:          blk.12.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  119:           blk.12.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  120:             blk.13.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  121:             blk.13.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  122:             blk.13.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  123:        blk.13.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  124:           blk.13.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  125:           blk.13.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  126:             blk.13.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  127:          blk.13.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  128:           blk.13.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  129:             blk.14.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  130:             blk.14.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  131:             blk.14.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  132:        blk.14.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  133:           blk.14.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  134:           blk.14.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  135:             blk.14.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  136:          blk.14.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  137:           blk.14.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  138:             blk.15.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  139:             blk.15.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  140:             blk.15.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  141:        blk.15.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  142:           blk.15.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  143:           blk.15.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  144:             blk.15.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  145:          blk.15.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  146:           blk.15.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  147:             blk.16.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  148:             blk.16.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  149:             blk.16.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  150:        blk.16.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  151:           blk.16.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  152:           blk.16.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  153:             blk.16.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  154:          blk.16.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  155:           blk.16.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  156:             blk.17.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  157:             blk.17.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  158:             blk.17.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  159:        blk.17.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  160:           blk.17.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  161:           blk.17.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  162:             blk.17.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  163:          blk.17.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  164:           blk.17.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  165:             blk.18.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  166:             blk.18.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  167:             blk.18.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  168:        blk.18.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  169:           blk.18.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  170:           blk.18.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  171:             blk.18.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  172:          blk.18.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  173:           blk.18.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  174:             blk.19.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  175:             blk.19.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  176:             blk.19.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  177:        blk.19.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  178:           blk.19.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  179:           blk.19.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  180:             blk.19.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  181:          blk.19.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  182:           blk.19.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  183:             blk.20.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  184:             blk.20.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  185:             blk.20.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  186:        blk.20.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  187:           blk.20.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  188:           blk.20.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  189:             blk.20.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  190:          blk.20.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  191:           blk.20.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  192:             blk.21.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  193:             blk.21.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  194:             blk.21.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  195:        blk.21.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  196:           blk.21.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  197:           blk.21.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  198:             blk.21.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  199:          blk.21.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  200:           blk.21.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  201:             blk.22.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  202:             blk.22.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  203:             blk.22.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  204:        blk.22.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  205:           blk.22.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  206:           blk.22.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  207:             blk.22.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  208:          blk.22.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  209:           blk.22.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  210:             blk.23.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  211:             blk.23.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  212:             blk.23.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  213:        blk.23.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  214:           blk.23.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  215:           blk.23.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  216:             blk.23.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  217:          blk.23.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  218:           blk.23.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  219:             blk.24.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  220:             blk.24.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  221:             blk.24.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  222:        blk.24.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  223:           blk.24.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  224:           blk.24.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  225:             blk.24.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  226:          blk.24.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  227:           blk.24.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  228:             blk.25.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  229:             blk.25.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  230:             blk.25.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  231:        blk.25.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  232:           blk.25.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  233:           blk.25.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  234:             blk.25.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  235:          blk.25.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  236:           blk.25.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  237:             blk.26.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  238:             blk.26.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  239:             blk.26.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  240:        blk.26.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  241:           blk.26.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  242:           blk.26.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  243:             blk.26.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  244:          blk.26.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  245:           blk.26.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  246:             blk.27.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  247:             blk.27.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  248:             blk.27.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  249:        blk.27.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  250:           blk.27.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  251:           blk.27.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  252:             blk.27.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  253:          blk.27.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  254:           blk.27.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  255:             blk.28.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  256:             blk.28.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  257:             blk.28.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  258:        blk.28.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  259:           blk.28.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  260:           blk.28.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  261:             blk.28.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  262:          blk.28.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  263:           blk.28.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  264:             blk.29.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  265:             blk.29.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  266:             blk.29.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  267:        blk.29.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  268:           blk.29.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  269:           blk.29.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  270:             blk.29.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  271:          blk.29.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  272:           blk.29.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  273:             blk.30.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  274:             blk.30.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  275:             blk.30.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  276:        blk.30.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  277:           blk.30.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  278:           blk.30.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  279:             blk.30.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  280:          blk.30.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  281:           blk.30.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  282:             blk.31.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  283:             blk.31.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  284:             blk.31.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  285:        blk.31.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  286:           blk.31.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  287:           blk.31.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  288:             blk.31.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  289:          blk.31.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  290:           blk.31.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  291:             blk.32.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  292:             blk.32.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  293:             blk.32.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  294:        blk.32.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  295:           blk.32.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  296:           blk.32.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  297:             blk.32.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  298:          blk.32.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  299:           blk.32.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  300:             blk.33.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  301:             blk.33.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  302:             blk.33.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  303:        blk.33.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  304:           blk.33.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  305:           blk.33.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  306:             blk.33.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  307:          blk.33.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  308:           blk.33.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  309:             blk.34.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  310:             blk.34.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  311:             blk.34.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  312:        blk.34.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  313:           blk.34.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  314:           blk.34.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  315:             blk.34.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  316:          blk.34.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  317:           blk.34.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  318:             blk.35.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  319:             blk.35.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  320:             blk.35.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  321:        blk.35.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  322:           blk.35.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  323:           blk.35.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  324:             blk.35.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  325:          blk.35.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  326:           blk.35.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  327:             blk.36.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  328:             blk.36.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  329:             blk.36.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  330:        blk.36.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  331:           blk.36.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  332:           blk.36.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  333:             blk.36.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  334:          blk.36.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  335:           blk.36.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  336:             blk.37.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  337:             blk.37.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  338:             blk.37.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  339:        blk.37.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  340:           blk.37.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  341:           blk.37.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  342:             blk.37.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  343:          blk.37.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  344:           blk.37.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  345:             blk.38.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  346:             blk.38.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  347:             blk.38.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  348:        blk.38.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  349:           blk.38.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  350:           blk.38.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  351:             blk.38.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  352:          blk.38.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  353:           blk.38.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  354:             blk.39.attn_q.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  355:             blk.39.attn_k.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  356:             blk.39.attn_v.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  357:        blk.39.attn_output.weight q4_0     [  5120,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  358:           blk.39.ffn_gate.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  359:           blk.39.ffn_down.weight q4_0     [ 13824,  5120,     1,     1 ]\n",
      "llama_model_loader: - tensor  360:             blk.39.ffn_up.weight q4_0     [  5120, 13824,     1,     1 ]\n",
      "llama_model_loader: - tensor  361:          blk.39.attn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: - tensor  362:           blk.39.ffn_norm.weight f32      [  5120,     1,     1,     1 ]\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = mostly Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size = 7024.03 MiB\n",
      "llm_load_tensors: mem required  = 7024.03 MiB\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "ggml_metal_init: allocating\n",
      "ggml_metal_init: found device: Apple M3\n",
      "ggml_metal_init: picking default device: Apple M3\n",
      "ggml_metal_init: default.metallib not found, loading from source\n",
      "ggml_metal_init: GGML_METAL_PATH_RESOURCES = nil\n",
      "ggml_metal_init: loading '/Users/xiongjiedai/llama.cpp/ggml-metal.metal'\n",
      "ggml_metal_init: GPU name:   Apple M3\n",
      "ggml_metal_init: GPU family: MTLGPUFamilyApple9 (1009)\n",
      "ggml_metal_init: hasUnifiedMemory              = true\n",
      "ggml_metal_init: recommendedMaxWorkingSetSize  = 11453.25 MB\n",
      "ggml_metal_init: maxTransferRate               = built-in GPU\n",
      "llama_new_context_with_model: compute buffer total size = 78.07 MiB\n",
      "llama_new_context_with_model: max tensor size =   128.17 MiB\n",
      "ggml_metal_add_buffer: allocated 'data            ' buffer, size =  7024.03 MiB, ( 7029.73 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'kv              ' buffer, size =   400.03 MiB, ( 7429.77 / 10922.67)\n",
      "ggml_metal_add_buffer: allocated 'alloc           ' buffer, size =    75.02 MiB, ( 7504.78 / 10922.67)\n",
      "\n",
      "system_info: n_threads = 4 / 8 | AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | SSSE3 = 0 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of its noble author, the French Revolution pursued its course and lighted up in every point of the compass that wild inhuman shadow which fell from it (although in the third volume it does not seem to be at all like a French novel, and I haven't read it for years). Various sensation novels were written on the subject: the \"Anticipations of Mrs. Shelley\" I should think must have been the worst of them; however, some other of them would be interesting if that one did not exist. The days of chivalry had expired; party conflicts were succeeded by contests for power. The feudal system was done away with, entailment abolished, foreshore and waste lands dealt with, and the world put into such a pitch of taking care of itself that there was nothing remaining to be done for it except pat it on the head, and say \"Well done!\" Accordingly, the patient was launched forth upon its way with every individual portion of its little human existence provided for, to all appearance. \n",
      " At this fortunate epoch, Mr. Jarndyce appearing one morning at Osborne Court, presented himself as ready to enter upon the great cause of his life; and he was not at all discomposed or cast down by anything that had since occurred, even in reference to those clients whose causes he had taken up.\n",
      " In a word, the cause was this:\n",
      " A certain person of upright character, great experience, and extensive observation, had fallen into some small mismanagement through want of practice (which continually strengthened his belief in these principles as applicable to the affairs of mankind), and into certain other mismanagement through want of foresight (which strengthened his belief in the necessity of preparing against emergencies). This person's name was Krook, and he was a solicitor. He was in request with Mr. Jarndyce, as one of those gentlemen who are always to be found when wanted, but seldom or never otherwise; whom you may engage in any suit without knowing whether they will come on your side or not, or what they will say or do if they do; who know all about it and will tell you nothing, and yet generally contrive to get in their clutches of the real legal practitioners, as a man with no particular objection to being scratched does, the fleas. He had been Mr. Jarndyce's agent for many years; he was so still: in fact, he could not have got rid of his master at any period, but at this time was so bound up with him that if a shark or a lion (which latter animal is much stronger) could by any means have divested Mr. Jarndyce of him, it would have been well for all concerned, and especially for Miss Flite.\n",
      " Krook's office had one door in Chancery-lane, which was always shut but never locked, and another door in an angle of the court of Chancer, near Temple-bar, by which he issued to go homeward after his day's work (and occasionally before it) on Saturdays and Sundays. He was a short man with a bald head, and a high colour—a peculiarity he had in common with the old gentleman whom we have seen already sitting behind a pair of spectacles at a certain window of Doctors' Commons: otherwise, in general appearance, nothing at all like that old gentleman, except that both of them were always complaining. Mr. Krook had never been heard to complain of being tired; but he was incessantly grumbling at the badness and meanness of things in general: very often he would stand with his hands in his pockets staring at a client like a cat at a mouse, until that client went away—whereat Mr. Krook never failed to mutter, 'Poor devil!' And once when Mr. Snagsby was passing the office door and looking in, he caught the words escape from Mr. Krook:\n",
      "'Rats!' said Mr. Krook, shaking his head slowly; 'rats! I'm tired of rats.'\n",
      "This was so much to the point that Mr. Snagsby repeated it several times as he walked on towards Lincoln's Inn, and still kept on repeating it at intervals long after dinner. And a very remarkable fact in connexion with the circumstance, is this: that it has since been found that there really are such things as rats in Doctors Commons!\n",
      "The door of Mr. Krook's office was never locked, because he never shut it, and if he had kept it always shut, the key would have been lost, long ago. The reason of this was, that there were\n",
      "llama_print_timings:        load time =    2622.60 ms\n",
      "llama_print_timings:      sample time =      70.41 ms /  1024 runs   (    0.07 ms per token, 14544.01 tokens per second)\n",
      "llama_print_timings: prompt eval time =    5085.31 ms /   494 tokens (   10.29 ms per token,    97.14 tokens per second)\n",
      "llama_print_timings:        eval time =   90334.41 ms /  1023 runs   (   88.30 ms per token,    11.32 tokens per second)\n",
      "llama_print_timings:       total time =   95613.54 ms\n",
      "ggml_metal_free: deallocating\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 1 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
